The notebook is composed of 4 parts.
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import matplotlib as mpl
import IPython.display as display
import PIL.Image
import functools
import time
import os
from numpy.random import seed
seed(111)
mpl.rcParams['figure.figsize'] = (15,15)
mpl.rcParams['axes.grid'] = False
def to_image(tensor):
tensor = tensor*255
tensor = np.array(tensor, dtype=np.uint8)
if np.ndim(tensor)>3:
assert tensor.shape[0] == 1
tensor = tensor[0]
return PIL.Image.fromarray(tensor)
content_path = 'columbia.jpg'
style_path = 'van_Gogh.jpg'
content_path2 = 'Manhattan.jpg'
style_path2 = 'Pablo_Picasso.jpg'
def load_img(path):
img = tf.io.read_file(path)
img = tf.image.decode_image(img, channels=3)
img = tf.image.convert_image_dtype(img, tf.float32)
shape = tf.cast(tf.shape(img)[:-1], tf.float32)
l = max(shape)
scale = 512 / l
new_shape = tf.cast(shape * scale, tf.int32)
img = tf.image.resize(img, new_shape)
img = img[tf.newaxis, :]
return img
content_image = load_img(content_path)
style_image = load_img(style_path)
x = tf.keras.applications.vgg19.preprocess_input(content_image*255)
x = tf.image.resize(x, (224, 224))
vgg = tf.keras.applications.VGG19(include_top=True, weights='imagenet')
vgg = tf.keras.applications.VGG19(include_top=False, weights='imagenet')
print()
for layer in vgg.layers:
print(layer.name)
# the layers to extract the content
content_layers = ['block4_conv2']
# the layers to extract the style
style_layers = ['block1_conv1',
'block2_conv1',
'block3_conv1',
'block4_conv1',
'block5_conv1']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
def vgg_layers(layers):
# load the pre-trained vgg
vgg = tf.keras.applications.VGG19(include_top=False, weights='imagenet')
vgg.trainable = False
output = [vgg.get_layer(name).output for name in layers]
model = tf.keras.Model([vgg.input], output)
return model
style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
def gram_matrix(x):
shape = tf.shape(x)
out1 = tf.linalg.einsum('bijc,bijd->bcd', x, x)
z = tf.cast(shape[1]*shape[2], tf.float32)
return out1/z
class NeuralTransfer(tf.keras.models.Model):
def __init__(self, style_layers, content_layers):
super(NeuralTransfer, self).__init__()
self.vgg = vgg_layers(style_layers + content_layers)
self.vgg.trainable = False
self.content_layers = content_layers
self.style_layers = style_layers
self.num_style_layers = len(style_layers)
def call(self, img):
"Expects float input in [0,1]"
img = img*255.0
img_pre = tf.keras.applications.vgg19.preprocess_input(img)
img1 = self.vgg(img_pre)
style = img1[:self.num_style_layers]
style = [gram_matrix(s) for s in style]
style_dict = {style_name:value
for style_name, value
in zip(self.style_layers, style)}
content = img1[self.num_style_layers:]
content_dict = {content_name:value
for content_name, value
in zip(self.content_layers, content)}
return {'style':style_dict,'content':content_dict}
extractor = NeuralTransfer(style_layers, content_layers)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
image = tf.Variable(content_image)
def image_clip(image):
return tf.clip_by_value(image, clip_value_min=0.0, clip_value_max=1.0)
opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)
style_weight=1
content_weight=1e-3
def compute_loss(x):
style = x['style']
style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2)
for i in style.keys()])
style_loss *= style_weight/num_style_layers
content = x['content']
content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2)
for i in content.keys()])
content_loss *= content_weight / num_content_layers
loss = style_loss + content_loss
return loss
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
train_iter(image)
train_iter(image)
train_iter(image)
to_image(image)
# train for 1000 interations
import time
import os
start = time.time()
epochs = 10
iter_per_epoch = 100
step = 0
for n in range(epochs):
for m in range(iter_per_epoch):
step += 1
train_iter(image)
save_path = os.path.join("img_%s.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("Train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
tf.random.set_seed(111)
content_image = load_img(content_path)
style_image = load_img(style_path)
plt.subplot(1, 2, 1)
show_image(content_image, 'Content Image')
plt.subplot(1, 2, 2)
show_image(style_image, 'Style Image')
style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
image_prepared = tf.Variable(content_image)
opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
style_weight=1
content_weight=1e-3
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
start = time.time()
epochs = 1000
image_prepared = basic_style_transfer_prepare(content_path, style_path)
save_path = 'img_c1_s1.jpg'
for n in range(epochs):
train_iter(image_prepared)
display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)
end = time.time()
print("Total time: {:.1f}".format(end-start))
content_image = load_img(content_path)
style_image = load_img(style_path2)
plt.subplot(1, 2, 1)
show_image(content_image, 'Content Image')
plt.subplot(1, 2, 2)
show_image(style_image, 'Style Image')
style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
image_prepared = tf.Variable(content_image)
opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
style_weight=1
content_weight=1e-3
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
start = time.time()
epochs = 1000
#image_prepared = basic_style_transfer_prepare(content_path, style_path)
save_path = 'img_c1_s2.jpg'
for n in range(epochs):
train_iter(image_prepared)
display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)
end = time.time()
print("Total time: {:.1f}".format(end-start))
content_image = load_img(content_path2)
style_image = load_img(style_path)
plt.subplot(1, 2, 1)
show_image(content_image, 'Content Image')
plt.subplot(1, 2, 2)
show_image(style_image, 'Style Image')
style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
image_prepared = tf.Variable(content_image)
opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
style_weight=1
content_weight=1e-3
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
start = time.time()
epochs = 1000
#image_prepared = basic_style_transfer_prepare(content_path, style_path)
save_path = 'img_c2_s1.jpg'
for n in range(epochs):
train_iter(image_prepared)
display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)
end = time.time()
print("Total time: {:.1f}".format(end-start))
content_image = load_img(content_path2)
style_image = load_img(style_path2)
plt.subplot(1, 2, 1)
show_image(content_image, 'Content Image')
plt.subplot(1, 2, 2)
show_image(style_image, 'Style Image')
style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
image_prepared = tf.Variable(content_image)
opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
style_weight=1
content_weight=1e-3
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
start = time.time()
epochs = 1000
save_path = 'img_c2_s2.jpg'
for n in range(epochs):
train_iter(image_prepared)
display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)
end = time.time()
print("Total time: {:.1f}".format(end-start))
basic_image = load_img("white.jpg")
# block1_conv1 : style
style_layers = ['block1_conv1']
extractor = NeuralTransfer(style_layers, content_layers)
results = extractor(tf.constant(basic_image))
style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
content_y = extractor(basic_image)['content']
style_y = extractor(style_image)['style']
image = tf.Variable(basic_image)
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
def compute_loss(x):
style = x['style']
style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2)
for i in style.keys()])
style_loss *= style_weight/num_style_layers
content = x['content']
content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2)
for i in content.keys()])
content_loss *= content_weight / num_content_layers
loss = content_loss + style_loss
return loss
step = 0
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("style_%s_1.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# block1_conv1 : style -> content
style_layers = ['block1_conv1']
extractor = NeuralTransfer(style_layers, content_layers)
results = extractor(tf.constant(content_image))
style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
def compute_loss(x):
style = x['style']
style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2)
for i in style.keys()])
style_loss *= style_weight/num_style_layers
content = x['content']
content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2)
for i in content.keys()])
content_loss *= content_weight / num_content_layers
loss = content_loss + style_loss
return loss
image = tf.Variable(content_image)
step = 0
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("content_%s_1.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# ['block1_conv1','block2_conv1']:style
style_layers = ['block1_conv1',
'block2_conv1']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
extractor = NeuralTransfer(style_layers, content_layers)
results = extractor(tf.constant(basic_image))
style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
content_y = extractor(basic_image)['content']
style_y = extractor(style_image)['style']
def compute_loss(x):
style = x['style']
style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2)
for i in style.keys()])
style_loss *= style_weight/num_style_layers
content = x['content']
content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2)
for i in content.keys()])
content_loss *= content_weight / num_content_layers
loss = content_loss + style_loss
return loss
image = tf.Variable(basic_image)
step = 0
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("style_%s_2.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# style -> content
style_layers = ['block1_conv1',
'block2_conv1']
extractor = NeuralTransfer(style_layers, content_layers)
results = extractor(tf.constant(content_image))
style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
def compute_loss(x):
style = x['style']
style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2)
for i in style.keys()])
style_loss *= style_weight/num_style_layers
content = x['content']
content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2)
for i in content.keys()])
content_loss *= content_weight / num_content_layers
loss = content_loss + style_loss
return loss
image = tf.Variable(content_image)
step = 0
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("content_%s_2.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# style
style_layers = ['block1_conv1',
'block2_conv1',
'block3_conv1']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
extractor = NeuralTransfer(style_layers, content_layers)
results = extractor(tf.constant(basic_image))
style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
content_y = extractor(basic_image)['content']
style_y = extractor(style_image)['style']
def compute_loss(x):
style = x['style']
style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2)
for i in style.keys()])
style_loss *= style_weight/num_style_layers
content = x['content']
content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2)
for i in content.keys()])
content_loss *= content_weight / num_content_layers
loss = content_loss + style_loss
return loss
image = tf.Variable(basic_image)
step = 0
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("style_%s_3.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# style -> content
style_layers = ['block1_conv1',
'block2_conv1',
'block3_conv1']
extractor = NeuralTransfer(style_layers, content_layers)
results = extractor(tf.constant(content_image))
style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
def compute_loss(x):
style = x['style']
style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2)
for i in style.keys()])
style_loss *= style_weight/num_style_layers
content = x['content']
content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2)
for i in content.keys()])
content_loss *= content_weight / num_content_layers
loss = content_loss + style_loss
return loss
image = tf.Variable(content_image)
step = 0
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("content_%s_3.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# block1_conv1, block2_conv1, block3_conv1, block4_conv1
# style
style_layers = ['block1_conv1',
'block2_conv1',
'block3_conv1',
'block4_conv1']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
extractor = NeuralTransfer(style_layers, content_layers)
results = extractor(tf.constant(basic_image))
style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
content_y = extractor(basic_image)['content']
style_y = extractor(style_image)['style']
def compute_loss(x):
style = x['style']
style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2)
for i in style.keys()])
style_loss *= style_weight/num_style_layers
content = x['content']
content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2)
for i in content.keys()])
content_loss *= content_weight / num_content_layers
loss = content_loss + style_loss
return loss
image = tf.Variable(basic_image)
step = 0
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("style_%s_4.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# style -> content
style_layers = ['block1_conv1',
'block2_conv1',
'block3_conv1',
'block4_conv1']
extractor = NeuralTransfer(style_layers, content_layers)
results = extractor(tf.constant(content_image))
style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
def compute_loss(x):
style = x['style']
style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2)
for i in style.keys()])
style_loss *= style_weight/num_style_layers
content = x['content']
content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2)
for i in content.keys()])
content_loss *= content_weight / num_content_layers
loss = content_loss + style_loss
return loss
image = tf.Variable(content_image)
step = 0
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("content_%s_4.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# style
style_layers = ['block1_conv1',
'block2_conv1',
'block3_conv1',
'block4_conv1',
'block5_conv1']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
extractor = NeuralTransfer(style_layers, content_layers)
results = extractor(tf.constant(basic_image))
style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
content_y = extractor(basic_image)['content']
style_y = extractor(style_image)['style']
image = tf.Variable(basic_image)
step = 0
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("style_%s_5.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# style -> content
style_layers = ['block1_conv1',
'block2_conv1',
'block3_conv1',
'block4_conv1',
'block5_conv1']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
extractor = NeuralTransfer(style_layers, content_layers)
results = extractor(tf.constant(content_image))
style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
def compute_loss(x):
style = x['style']
style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2)
for i in style.keys()])
style_loss *= style_weight/num_style_layers
content = x['content']
content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2)
for i in content.keys()])
content_loss *= content_weight / num_content_layers
loss = content_loss + style_loss
return loss
image = tf.Variable(content_image)
step = 0
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("content_%s_5.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
Iterations = [10, 50, 100, 500, 1000, 5000, 10000,12000,14000,15000]
style_layers = ['block1_conv1',
'block2_conv1',
'block3_conv1',
'block4_conv1',
'block5_conv1']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
extractor = NeuralTransfer(style_layers, content_layers)
results = extractor(tf.constant(content_image))
style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
image = tf.Variable(content_image)
start = time.time()
step = 0
for i in range(15000):
step += 1
train_iter(image)
if step in [10, 50, 100, 500, 1000, 5000, 10000,12000,14000,15000]:
save_path = os.path.join("img_%s.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("current time: {:.1f}".format(end-start))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# lr = 0.01
content_image = load_img(content_path)
style_image = load_img(style_path)
random_seed = 111
tf.random.set_seed(random_seed)
style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
image_prepared = tf.Variable(content_image)
opt = tf.optimizers.Adam(learning_rate=0.01, beta_1=0.99, epsilon=1e-1)
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
style_weight=1
content_weight=1e-3
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
step = 0
image = tf.Variable(content_image)
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("img_%s_0.01.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# lr = 0.05
content_image = load_img(content_path)
style_image = load_img(style_path)
random_seed = 111
style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
tf.random.set_seed(random_seed)
opt = tf.optimizers.Adam(learning_rate=0.05, beta_1=0.99, epsilon=1e-1)
image_prepared = tf.Variable(content_image)
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
style_weight=1
content_weight=1e-3
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
step = 0
image = tf.Variable(content_image)
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("img_%s_0.05.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# lr = 0.1
content_image = load_img(content_path)
style_image = load_img(style_path)
random_seed = 111
style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
tf.random.set_seed(random_seed)
opt = tf.optimizers.Adam(learning_rate=0.1, beta_1=0.99, epsilon=1e-1)
image_prepared = tf.Variable(content_image)
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
style_weight=1
content_weight=1e-3
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
step = 0
image = tf.Variable(content_image)
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("img_%s_0.1.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# lr = 0.01
content_image = load_img(content_path)
style_image = load_img(style_path)
random_seed = 111
style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
tf.random.set_seed(random_seed)
opt = tf.optimizers.Adam(learning_rate=0.01, beta_1=0.99, epsilon=1e-1)
image_prepared = tf.Variable(content_image)
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
style_weight=1
content_weight=1e-3
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
step = 0
image = tf.Variable(content_image)
start = time.time()
for i in range(2000):
step += 1
train_iter(image)
save_path = os.path.join("img_%s_0.01.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# lr = 0.05
content_image = load_img(content_path)
style_image = load_img(style_path)
random_seed = 111
style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
tf.random.set_seed(random_seed)
opt = tf.optimizers.Adam(learning_rate=0.05, beta_1=0.99, epsilon=1e-1)
image_prepared = tf.Variable(content_image)
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
style_weight=1
content_weight=1e-3
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
step = 0
image = tf.Variable(content_image)
start = time.time()
for i in range(2000):
step += 1
train_iter(image)
save_path = os.path.join("img_%s_0.05.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# lr = 0.10
content_image = load_img(content_path)
style_image = load_img(style_path)
random_seed = 111
style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
tf.random.set_seed(random_seed)
opt = tf.optimizers.Adam(learning_rate=0.1, beta_1=0.99, epsilon=1e-1)
image_prepared = tf.Variable(content_image)
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
style_weight=1
content_weight=1e-3
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
step = 0
image = tf.Variable(content_image)
start = time.time()
for i in range(2000):
step += 1
train_iter(image)
save_path = os.path.join("img_%s_0.1.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# the layers to extract the content
content_layers = ['block5_conv2']
# the layers to extract the style
style_layers = ['block1_conv1',
'block2_conv1',
'block3_conv1',
'block4_conv1',
'block5_conv1']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
extractor = NeuralTransfer(style_layers, content_layers)
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
image = tf.Variable(content_image)
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
opt = tf.optimizers.Adam(learning_rate=0.05, beta_1=0.99, epsilon=1e-1)
start = time.time()
epochs = 5
iter_per_epoch = 200
step = 0
for n in range(epochs):
for m in range(iter_per_epoch):
step += 1
train_iter(image)
print(".", end='')
display.clear_output(wait=True)
display.display(to_image(image),save_path)
print("Train step: {}".format(step))
save_path = os.path.join("img_%s_cont5-2.png"%(step))
to_image(image).save(save_path)
end = time.time()
print("Total time: {:.1f}".format(end-start))
# the layers to extract the content
content_layers = ['block4_conv2']
# the layers to extract the style
style_layers = ['block1_conv2',
'block2_conv1',
'block3_conv1',
'block4_conv1',
'block5_conv1']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
extractor = NeuralTransfer(style_layers, content_layers)
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
image = tf.Variable(content_image)
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)
start = time.time()
epochs = 5
iter_per_epoch = 200
step = 0
for n in range(epochs):
for m in range(iter_per_epoch):
step += 1
train_iter(image)
print(".", end='')
display.clear_output(wait=True)
display.display(to_image(image),save_path)
print("Train step: {}".format(step))
save_path = os.path.join("img_%s_sty1-2.png"%(step))
to_image(image).save(save_path)
end = time.time()
print("Total time: {:.1f}".format(end-start))
def compute_loss_with_ratio(x, content_weight, style_weight):
style = x['style']
style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2)
for i in style.keys()])
style_loss *= style_weight/num_style_layers
content = x['content']
content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2)
for i in content.keys()])
content_loss *= content_weight / num_content_layers
loss = style_loss + content_loss
return loss
@tf.function()
def train_iter_with_ratio(image, content_weight, style_weight):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss_with_ratio(x, content_weight, style_weight)
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
# alpha: content, beta: style
# alpha/beta: 1e-4
content_image = load_img(content_path)
image_prepared = tf.Variable(content_image)
content_weight=1e-4
style_weight=1
@tf.function()
def train_iter_with_ratio(image, content_weight, style_weight):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss_with_ratio(x, content_weight, style_weight)
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
start = time.time()
epochs = 1000
save_path = 'img_weight_1e-4.jpg'
for n in range(epochs):
train_iter_with_ratio(image_prepared, content_weight, style_weight)
display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)
end = time.time()
print("Total time: {:.1f}".format(end-start))
# alpha: content, beta: style
# alpha/beta: 1e-2
content_image = load_img(content_path)
image_prepared = tf.Variable(content_image)
content_weight=1e-2
style_weight=1
@tf.function()
def train_iter_with_ratio(image, content_weight, style_weight):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss_with_ratio(x, content_weight, style_weight)
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
start = time.time()
epochs = 1000
save_path = 'img_weight_1e-2.jpg'
for n in range(epochs):
train_iter_with_ratio(image_prepared, content_weight, style_weight)
display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)
end = time.time()
print("Total time: {:.1f}".format(end-start))
# alpha: content, beta: style
# alpha/beta: 1e2
content_image = load_img(content_path)
image_prepared = tf.Variable(content_image)
content_weight=1e2
style_weight=1
@tf.function()
def train_iter_with_ratio(image, content_weight, style_weight):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss_with_ratio(x, content_weight, style_weight)
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
start = time.time()
epochs = 1000
save_path = 'img_weight_1e+2.jpg'
for n in range(epochs):
train_iter_with_ratio(image_prepared, content_weight, style_weight)
display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)
end = time.time()
print("Total time: {:.1f}".format(end-start))
# alpha: content, beta: style
# alpha/beta: 1e4
content_image = load_img(content_path)
image_prepared = tf.Variable(content_image)
content_weight=1e4
style_weight=1
@tf.function()
def train_iter_with_ratio(image, content_weight, style_weight):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss_with_ratio(x, content_weight, style_weight)
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
start = time.time()
epochs = 1000
save_path = 'img_weight_1e+4.jpg'
for n in range(epochs):
train_iter_with_ratio(image_prepared, content_weight, style_weight)
display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)
end = time.time()
print("Total time: {:.1f}".format(end-start))
seed = [111,222,333,999]
# seed = 111
random_seed = 111
tf.random.set_seed(random_seed)
opt = tf.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1)
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
step = 0
image = tf.Variable(content_image)
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("img_%s_111.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
# seed = 222
tf.random.set_seed(222)
opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
image = tf.Variable(content_image)
step = 0
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("img_%s_222.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
tf.random.set_seed(333)
opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
image = tf.Variable(content_image)
step = 0
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("img_%s_333.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
tf.random.set_seed(999)
opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
step = 0
start = time.time()
for i in range(1000):
step += 1
train_iter(image)
save_path = os.path.join("img_%s_999.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))
end = time.time()
print("Total time: {:.1f}".format(end-start))
def load_img_w2(path):
img = tf.io.read_file(path)
img = tf.image.decode_image(img, channels=3)
img = tf.image.convert_image_dtype(img, tf.float32)
new_shape = tf.cast([224,224], tf.int32)
img = tf.image.resize(img, new_shape)
img = img[tf.newaxis, :]
return img
content_image_w2 = load_img_w2(content_path)
style_image_w2 = load_img_w2(style_path)
# the layers to extract the content
content_layers = ['block4_conv2']
# the layers to extract the style
style_layers = ['block1_conv1',
'block2_conv1',
'block3_conv1',
'block4_conv1',
'block5_conv1']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
def vgg_layers_w2(layers):
# load the pre-trained vgg
vgg = tf.keras.applications.VGG19()
vgg.load_weights("vgg19_weights_tf_dim_ordering_tf_kernels.h5")
vgg.trainable = False
output = [vgg.get_layer(name).output for name in layers]
model = tf.keras.Model([vgg.input], output)
return model
style_extractor = vgg_layers_w2(style_layers)
style_outputs = style_extractor(style_image_w2*255)
'''
#查看每层输出的统计信息
for name, output in zip(style_layers, style_outputs):
print(name)
print(" shape: ", output.numpy().shape)
print(" min: ", output.numpy().min())
print(" max: ", output.numpy().max())
print(" mean: ", output.numpy().mean())
print()
'''
class NeuralTransfer_w2(tf.keras.models.Model):
def __init__(self, style_layers, content_layers):
super(NeuralTransfer_w2, self).__init__()
self.vgg = vgg_layers_w2(style_layers + content_layers)
self.vgg.trainable = False
self.content_layers = content_layers
self.style_layers = style_layers
self.num_style_layers = len(style_layers)
def call(self, img):
"Expects float input in [0,1]"
img = img*255.0
img_pre = tf.keras.applications.vgg19.preprocess_input(img)
img1 = self.vgg(img_pre)
style = img1[:self.num_style_layers]
style = [gram_matrix(s) for s in style]
style_dict = {style_name:value
for style_name, value
in zip(self.style_layers, style)}
content = img1[self.num_style_layers:]
content_dict = {content_name:value
for content_name, value
in zip(self.content_layers, content)}
return {'style':style_dict,'content':content_dict}
extractor = NeuralTransfer_w2(style_layers,content_layers)
content_y = extractor(content_image_w2)['content']
style_y = extractor(style_image_w2)['style']
image_w2 = tf.Variable(content_image_w2)
@tf.function()
def train_iter(image_w2):
with tf.GradientTape() as tape:
x = extractor(image_w2)
loss = compute_loss(x)
grad = tape.gradient(loss, image_w2)
opt.apply_gradients([(grad, image_w2)])
image_w2.assign(image_clip(image_w2))
opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)
start = time.time()
epochs = 5
iter_per_epoch = 200
step = 0
for n in range(epochs):
for m in range(iter_per_epoch):
step += 1
train_iter(image_w2)
print(".", end='')
display.clear_output(wait=True)
shape_w2 = tf.cast([274, 365], tf.int32)
img_w2 = tf.image.resize(image_w2,shape_w2)
display.display(to_image(img_w2))
print("Train step: {}".format(step))
save_path = os.path.join("img_%s_w2.png"%(step))
to_image(img_w2).save(save_path)
end = time.time()
print("Total time: {:.1f}".format(end-start))
content_image = load_img(content_path2)
style_image = load_img(content_path)
plt.subplot(1, 2, 1)
show_image(content_image, 'Content Image')
plt.subplot(1, 2, 2)
show_image(style_image, 'Style Image')
style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
image_prepared = tf.Variable(content_image)
opt = tf.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1)
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
style_weight=1
content_weight=1
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
start = time.time()
epochs = 1000
#image_prepared = basic_style_transfer_prepare(content_path, style_path)
save_path = 'img_c2_c1.jpg'
for n in range(epochs):
train_iter(image_prepared)
display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)
end = time.time()
print("Total time: {:.1f}".format(end-start))
content_image = load_img(style_path)
style_image = load_img(style_path2)
plt.subplot(1, 2, 1)
show_image(content_image, 'Content Image')
plt.subplot(1, 2, 2)
show_image(style_image, 'Style Image')
style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
image_prepared = tf.Variable(content_image)
opt = tf.optimizers.Adam(learning_rate=0.01, epsilon=1e-1)
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
style_weight=1
content_weight=1
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
start = time.time()
epochs = 1000
#image_prepared = basic_style_transfer_prepare(content_path, style_path)
save_path = 'img_s1_s2.jpg'
for n in range(epochs):
train_iter(image_prepared)
display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)
end = time.time()
print("Total time: {:.1f}".format(end-start))
style_path3 = 'Claude_Monet_57.jpg'
content_image = load_img(style_path)
style_image = load_img(style_path3)
plt.subplot(1, 2, 1)
show_image(content_image, 'Content Image')
plt.subplot(1, 2, 2)
show_image(style_image, 'Style Image')
style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
image_prepared = tf.Variable(content_image)
opt = tf.optimizers.Adam(learning_rate=0.01, beta_1=0.99, epsilon=1e-1)
@tf.function()
def train_iter(image):
with tf.GradientTape() as tape:
x = extractor(image)
loss = compute_loss(x)
style_weight=1
content_weight=10
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(image_clip(image))
start = time.time()
epochs = 1000
#image_prepared = basic_style_transfer_prepare(content_path, style_path)
save_path = 'img_s1_s3.jpg'
for n in range(epochs):
train_iter(image_prepared)
display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)
end = time.time()
print("Total time: {:.1f}".format(end-start))